{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.sparse\n",
    "from sklearn.decomposition import PCA,NMF,SparsePCA,TruncatedSVD,LatentDirichletAllocation\n",
    "import scipy.stats as sps\n",
    "from scipy.sparse import vstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seed = 1024\n",
    "np.random.seed(seed)\n",
    "\n",
    "path = '../data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_pickle(path + \"train_context_tfidf.pkl\")\n",
    "valid = pd.read_pickle(path + \"valid_context_tfidf.pkl\")\n",
    "dev = pd.read_pickle(path+\"dev_context_tfidf.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_all = vstack((train,valid,dev)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nmf = NMF(n_components=12,random_state=1123)\n",
    "nmf.fit(data_all)\n",
    "nmf_fea = nmf.components_.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30920, 12)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmf_fea.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_tfidf = nmf_fea[:train.shape[0]]\n",
    "valid_tfidf = nmf_fea[train.shape[0]:(train.shape[0]+valid.shape[0])]\n",
    "dev_tfidf = nmf_fea[(train.shape[0]+valid.shape[0]):]\n",
    "\n",
    "f = 'nmf'\n",
    "pd.to_pickle(train_tfidf, path + 'train_%s_tfidf.pkl' % f)\n",
    "pd.to_pickle(valid_tfidf,path+'valid_%s_tfidf.pkl'%f)\n",
    "pd.to_pickle(dev_tfidf, path + 'dev_%s_tfidf.pkl' % f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PCA主题模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#pca\n",
    "svd = TruncatedSVD(n_components=12,random_state=1123)\n",
    "svd.fit(data_all)\n",
    "svd_fea = svd.components_.T\n",
    "\n",
    "train_tfidf = svd_fea[:train.shape[0]]\n",
    "valid_tfidf = svd_fea[train.shape[0]:(train.shape[0]+valid.shape[0])]\n",
    "dev_tfidf = svd_fea[(train.shape[0]+valid.shape[0]):]\n",
    "\n",
    "f = 'svd'\n",
    "pd.to_pickle(train_tfidf, path + 'train_%s_tfidf.pkl' % f)\n",
    "pd.to_pickle(valid_tfidf,path+'valid_%s_tfidf.pkl'%f)\n",
    "pd.to_pickle(dev_tfidf, path + 'dev_%s_tfidf.pkl' % f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LDA主题模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_topics=5,max_iter=5,random_state=1123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda\\lib\\site-packages\\sklearn\\decomposition\\online_lda.py:508: DeprecationWarning: The default value for 'learning_method' will be changed from 'online' to 'batch' in the release 0.20. This warning was introduced in 0.18.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "lda.fit(data_all)\n",
    "lda_fea = lda.components_.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
